package zemberek3.crosscheck;

import com.google.common.base.Joiner;
import com.google.common.base.Splitter;
import com.google.common.collect.*;
import org.jcaki.Files;
import org.jcaki.SimpleTextReader;
import org.simpleframework.xml.Attribute;
import org.simpleframework.xml.ElementList;
import org.simpleframework.xml.Root;
import org.simpleframework.xml.core.Persister;

import java.io.File;
import java.io.IOException;
import java.text.Collator;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class ValidationDictionaryGenerator {
    public static final Comparator<File> FILE_COMPARATOR = Files.getNameSortingComparator(new Locale("tr"));
    File htmlRoot;
    List<File> dirs = new ArrayList<File>();

    public ValidationDictionaryGenerator(File htmlRoot) {
        this.htmlRoot = htmlRoot;
        dirs = Files.getDirectories(htmlRoot, false);
        Collections.sort(dirs, FILE_COMPARATOR);
    }

    static Pattern CONTENT_CHUNK_PATTERN = Pattern.compile("(<table width='630' border=0  id='hor-minimalist-a'>)(.+?)(</table>)", Pattern.DOTALL | Pattern.MULTILINE);

    static Pattern HEADER_PATTERN = Pattern.compile("(?:<th scope='col'>)(.+?)(?:</th>)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern NAME_PATTERN = Pattern.compile("(?:<b>)(.+?)(?:</b>)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern TYPE_PATTERN = Pattern.compile("(?:<i><b>)(.+?)(?:</b>)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern ATTR_PATTERN = Pattern.compile("(?:</b>)(.+?)(?:<br>)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern TYPE_GROUP_PATTERN = Pattern.compile("(?:<i>)(.+?)(?:$)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern ORIGIN_PATTERN = Pattern.compile("(?:</b>)(.+?)(?:</i>$)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern PRONUNCIATION_PATTERN = Pattern.compile("(?:\\()(.+?)(?:\\))", Pattern.DOTALL | Pattern.MULTILINE);

    static Pattern MEANING_CHUNK_PATTERN = Pattern.compile("(?:<tr>[ ]+<td>)(.+?)(?:</td>)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern EXAMPLE_PATTERN = Pattern.compile("(?:\"<i>)(.+?)(?:</i>)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern EXAMPLE_AUTHOR_PATTERN = Pattern.compile("(?:<b>)(.+?)(?:</b>)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern MEANING_TYPE_PATTERN = Pattern.compile("(?:[0-9]+[.][ ]+<i>)(.+?)(?:</i> )", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern MEANING_PATTERN = Pattern.compile("(?:</i> )(.+?)(?:<br>)", Pattern.DOTALL | Pattern.MULTILINE);
    static Pattern VERB_ATTR_PATTERN = Pattern.compile("nsz|[-][^ ]+", Pattern.DOTALL | Pattern.MULTILINE);


    private static List<String> group1Matches(String s, Pattern p) {
        Matcher m = p.matcher(s);
        List<String> res = new ArrayList<String>();
        while (m.find()) {
            res.add(m.group(1).trim());
        }
        return res;
    }

    private static String group1Match(String s, Pattern p) {
        Matcher m = p.matcher(s);
        if (m.find()) {
            return m.group(1).trim();
        }
        return "";
    }

    private static List<String> matches(String s, Pattern p) {
        Matcher m = p.matcher(s);
        List<String> res = new ArrayList<String>();
        while (m.find()) {
            res.add(m.group().trim());
        }
        return res;
    }

    List<DictionaryUnit> collect(int itemCount) throws IOException {

        Multimap<MainInfo, SubUnit> units = HashMultimap.create();

        for (File dir : dirs) {

            System.out.println(dir);
            List<File> files = Files.crawlDirectory(dir, false, Files.extensionFilter("html"));
            Collections.sort(files, FILE_COMPARATOR);
            for (File file : files) {
                String allContent = SimpleTextReader.trimmingUTF8Reader(file).asString();
                allContent = allContent.replaceAll("\\s", " ");
                List<String> contentChunks = matches(allContent, CONTENT_CHUNK_PATTERN);

                for (String contentChunk : contentChunks) {
                    String header = group1Match(contentChunk, HEADER_PATTERN);

                    MainInfo info = new MainInfo(header);
                    Multimap<String, Meaning> typez = ArrayListMultimap.create();
                    // get meaning lines
                    List<String> meaningLines = group1Matches(contentChunk, MEANING_CHUNK_PATTERN);

                    for (String meaningLine : meaningLines) {

                        String meaningType = group1Match(meaningLine, MEANING_TYPE_PATTERN);
                        String type = info.pos;
                        List<String> annotations = info.annotations;
                        if (meaningType != null && meaningType.length() > 0) {
                            for (String token : Splitter.on(",").omitEmptyStrings().trimResults().split(meaningType)) {
                                if (TYPES.contains(token))
                                    type = token;
                                if (ANNOTATIONS.contains(token)) {
                                    if (!annotations.contains(token))
                                        annotations = Lists.newArrayList(token);
                                }
                            }
                        }
                        String content = group1Match(meaningLine, MEANING_PATTERN);
                        String example = group1Match(meaningLine, EXAMPLE_PATTERN);
                        String exampleAuthor = group1Match(meaningLine, EXAMPLE_AUTHOR_PATTERN);

                        typez.put(type, new Meaning(content, example, exampleAuthor, annotations));
                    }

                    for (String type : typez.keySet()) {
                        SubUnit subUnit = new SubUnit(type, Lists.newArrayList(typez.get(type)));
                        units.put(info, subUnit);
                    }


                }
                if (units.size() > itemCount)
                    break;
            }
        }

        List<DictionaryUnit> dictionaryUnits = Lists.newArrayList();
        for (MainInfo info : units.keySet()) {
            dictionaryUnits.add(new DictionaryUnit(info, Lists.newArrayList(units.get(info))));
        }
        return dictionaryUnits;
    }

    public static Set<String> TYPES = Sets.newHashSet("isim", "sıfat", "fiil", "zamir", "zarf", "ünlem", "bağlaç", "edat");
    public static Set<String> ANNOTATIONS = Sets.newHashSet(
            "özel", "eskimiş", "felsefe", "ticaret", "eğitim bilimi", "antropoloji",
            "toplum bilimi", "mineraloji", "dil bilgisi", "halk ağzında", "coğrafya", "TV", "argo", "mantık", "anatomi", "biyoloji",
            "bilişim", "alay yollu", "gök bilimi", "kaba konuşmada", "şaka yollu", "spor", "matematik", "hayvan bilimi",
            "bitki bilimi", "sinema", "ruh bilimi", "din b.", "teklifsiz konuşmada", "hakaret yollu", "tıp", "tiyatro", "mecaz", "mimarlık",
            "hukuk", "kimya", "edebiyat", "müzik", "tarih", "ekonomi", "askerlik", "denizcilik", "jeoloji", "fizik", "dil bilimi", "madencilik",
            "meteoroloji", "teknik", "geometri", "fizyoloji");
    public static Set<String> ATTRIBUTES = Sets.newHashSet("-i", "-a", "nsz");

    public static Set<String> DROPS = Sets.newHashSet(
            "-brı", "-yti", "-cmi", "-drı", "-ynı", "-ğlu", "-hrü", "-zfi",
            "-tri", "-kşı", "-tbu", "-yfi", "-zmı", "-mrü", "-smi", "-hvi", "-çhi", "-fni", "-psi", "-khı", "-cmi", "-slü", "-vrı", "-zrü", "-hsı",
            "-sfı", "-vli", "-tfı", "-fyi", "-nzi", "-sli", "-hyi", "-tli", "-lsü", "-znü", "-hrı", "-ynu", "-tku", "-yni", "-zri",
            "-ptı", "-zbi", "-kli", "-thı", "-ğrü", "-şfi", "-mli", "-yzı", "-rnu", "-tni", "-bli", "-vmi", "-lmi", "-ğsü", "-mzu", "-mzi", "-hzi",
            "-rmi", "-kzi", "-stı", "-kfı", "-tmi", "-sbı", "-hli", "-thi", "-shi", "-bzı", "-zmi", "-yzi", "-hvı", "-şrü", "-ydı",
            "-hmi", "-lfü", "-dri", "-trı", "-ğzı", "-bri", "-hşu", "-zni", "-şvi", "-srı", "-cri", "-hni", "-frü", "-vri", "-ğru", "-şyi", "-krü",
            "-şrı", "-rmü", "-yfı", "-yli", "-nlü", "-zli", "-ğvı", "-smı", "-rnı", "-şmı", "-cvi", "-fzı", "-tnı", "-sci", "-şri", "-kti", "-vci",
            "-knü", "-msi", "-dli", "-klı", "-sri", "-tfu", "-ksi", "-hdi", "-hri", "-kdi", "-mri", "-kmü", "-yrı", "-slı", "-lmü", "-snü", "-fsi",
            "-hsi", "-kri", "-ybı", "-lnı", "-kzı", "-fku"
    );

    public static Set<String> REPEAT = Sets.newHashSet(
            "-bbı", "-ffı", "-ddı", "-nni", "-kki", "-lli", "-ssı", "-ddi",
            "-zzı", "-yyı", "-mmi", "-ssü", "-nnü", "-ffi", "-rri", "-nnı",
            "-hhı", "-ttı", "-llü", "-mmı", "-kkı", "-ssi", "-ccı", "-rrı",
            "-ddı"
    );


    /*
        List<MainInfo> getTypeInfo(String s) {
            List<MainInfo> types = new ArrayList<MainInfo>();
            s = s.replaceAll("-[^ ][ ]+", " ").trim();
        }
    */
    public static Multimap<String, String> attrs = HashMultimap.create();

    @Root(name = "mn")
    static class Meaning {
        @Attribute
        public String content;

        @Attribute(required = false, name = "ex")
        public String example;

        @Attribute(required = false, name = "ex-src")
        public String exampleSource;

        @ElementList(inline = true, required =false, entry = "an")
        public List<String> annotations = Lists.newArrayList();

        public Meaning() {
        }

        Meaning(String content, String example, String exampleSource ,List<String> annotations) {
            this.content = content;
            this.example = example.length() == 0 ? null : example;
            this.exampleSource = exampleSource.length() == 0 ? null : exampleSource;
            this.annotations = annotations;
        }

        public String toString() {
            StringBuilder sb = new StringBuilder();
            if (content != null)
                sb.append("Mn:").append(content);
            if (example != null && example.length() != 0)
                sb.append("; Ex:").append(example);
            if (exampleSource != null && exampleSource.length() != 0)
                sb.append(" (").append(exampleSource).append(")");
            return sb.toString().trim();
        }
    }

    @Root(name = "dictionary")
    static class TdkDictionary {
        @ElementList(inline = true)
        public List<DictionaryUnit> units = Lists.newArrayList();

        public static Collator trColl = Collator.getInstance(new Locale("tr"));

        TdkDictionary() {
        }

        TdkDictionary(List<DictionaryUnit> units) {
            this.units = units;
        }

        void sort() {
            Collections.sort(units, new Comparator<DictionaryUnit>() {
                @Override
                public int compare(DictionaryUnit o1, DictionaryUnit o2) {
                    return trColl.compare(o1.name, o2.name);
                }
            });
        }

        public void saveXml(File file) throws Exception {
            sort();
            new Persister().write(this, file);
        }

        public static TdkDictionary loadXml(File file) {
            try {
                return new Persister().read(TdkDictionary.class, file);
            } catch (Exception e) {
                e.printStackTrace();
                return null;
            }
        }

    }

    @Root(name = "unit")
    static class DictionaryUnit {

        @Attribute
        public String name;

        @Attribute(required = false)
        public String origin;

        @Attribute(required = false, name = "pron")
        public String pronunciation;

        @Attribute(required = false)
        public String attributes;

        @ElementList(inline = true, name = "sub")
        public List<SubUnit> subUnits = Lists.newArrayList();

        public DictionaryUnit(){
        }

        DictionaryUnit(MainInfo info, List<SubUnit> subUnits) {
            this.name = info.name;
            this.origin = info.origin.length() == 0 ? null : info.origin;
            this.pronunciation = info.pronunciation.length() == 0 ? null : info.pronunciation;
            String atr = Joiner.on(" ").join(info.attributes);
            this.attributes = atr.length() == 0 ? null : atr;
            this.subUnits = subUnits;
        }

        void add(SubUnit subUnit) {
            subUnits.add(subUnit);
        }

    }

    @Root(name = "sub")
    static class SubUnit {

        @Attribute
        public String pos;

        @ElementList(inline = true)
        public List<Meaning> meanings = Lists.newArrayList();

        SubUnit() {
        }

        SubUnit(String pos, List<Meaning> meanings) {
            this.pos = pos;
            this.meanings = meanings;
        }
    }

    static class MainInfo {
        public String name;
        public String pos;
        public String origin;
        public String pronunciation;
        public List<String> annotations = Lists.newArrayList();
        public List<String> attributes = Lists.newArrayList();

        MainInfo() {
        }

        MainInfo(String header) {
            name = group1Match(header, NAME_PATTERN).replaceAll("\\([^\\)]+\\)", " ").trim();
            String typeBlock = group1Match(header, TYPE_PATTERN);
            pronunciation = group1Match(typeBlock, PRONUNCIATION_PATTERN); // pronunciation is in paranthesis
            String typeData = typeBlock.replaceAll("\\([^\\)]+\\)", " ").replaceAll("[, ]+", " ").trim(); // replace paranthesis content
            String attrData = group1Match(header, ATTR_PATTERN);
            attrs.put(attrData, name);
            if (typeData.contains("-") || typeData.contains("nsz")) {
                typeData = "fiil " + typeData.replaceAll("-[^ ]+|nsz", " ").trim();
                attrData += Joiner.on(",").join(matches(typeData, VERB_ATTR_PATTERN));
            }
            origin = group1Match(group1Match(header, TYPE_GROUP_PATTERN), ORIGIN_PATTERN);
            setTypeAndAnotations(typeData);
        }

        @Override
        public boolean equals(Object o) {
            if (this == o) return true;
            if (o == null || getClass() != o.getClass()) return false;

            MainInfo mainInfo = (MainInfo) o;

            if (!annotations.equals(mainInfo.annotations)) return false;
            if (!attributes.equals(mainInfo.attributes)) return false;
            if (!name.equals(mainInfo.name)) return false;
            if (!origin.equals(mainInfo.origin)) return false;
            if (!pos.equals(mainInfo.pos)) return false;
            if (!pronunciation.equals(mainInfo.pronunciation)) return false;

            return true;
        }

        @Override
        public int hashCode() {
            int result = name.hashCode();
            result = 31 * result + pos.hashCode();
            result = 31 * result + annotations.hashCode();
            result = 31 * result + pronunciation.hashCode();
            result = 31 * result + attributes.hashCode();
            result = 31 * result + origin.hashCode();
            return result;
        }

        public void setTypeAndAnotations(String typeData) {
            boolean typeFound = false;
            for (String type : TYPES) {
                if (typeData.contains(type)) {
                    if (typeFound) {
                        System.out.println("Double type for:" + name + " type=" + pos + " dup=" + type);
                    } else pos = type;
                    typeData = typeData.replace(type, "").trim();
                    typeFound = true;

                }
            }
            if (!typeFound) {
                if (Character.isUpperCase(name.charAt(0))) {
                    if (!annotations.contains("özel"))
                        annotations.add("özel");
                    pos = "isim";
                } else if (name.endsWith("mek") || name.endsWith("mak")) {
                    pos = "fiil";
                } else pos = "isim";
            }
            for (String annotation : ANNOTATIONS) {
                if (typeData.contains(annotation)) {
                    typeData = typeData.replace(annotation, "").trim();
                    if (!annotations.contains(annotation))
                        annotations.add(annotation);
                }
            }

            typeData = typeData.replaceAll("[ ,]+", "");

            if (typeData.length() > 0) {
                attrs.put(typeData, name);
            }
        }

        private String annotationString() {
            if (annotations.size() == 0)
                return "";
            return " An:" + Joiner.on(", ").join(annotations);
        }

        private String attrString() {
            if (attributes.size() == 0)
                return "";
            return " A:" + Joiner.on(", ").join(attributes);
        }

        public String asCompactString() {
            return name + " [P:" + pos + attrString() + annotationString() + "]";
        }

        public MainInfo copyForType(String typeData) {
            MainInfo m = new MainInfo();
            m.name = name;
            m.pos = pos;
            m.annotations = Lists.newArrayList(annotations);
            m.attributes = Lists.newArrayList(attributes);
            m.pronunciation = pronunciation;
            m.origin = origin;
            m.setTypeAndAnotations(typeData);
            return m;
        }
    }

    public static void main(String[] args) throws Exception {
        ValidationDictionaryGenerator gen = new ValidationDictionaryGenerator(new File("/home/kodlab/data/tdk/html"));
        List<DictionaryUnit> units = gen.collect(100000);
        TdkDictionary dictionary = new TdkDictionary(units);
        dictionary.saveXml(new File("/home/kodlab/data/lm/unknowns/tdk-dic.xml"));
    }

    public static TdkDictionary getDictionaryFromXml(File xmlFile) {
        return TdkDictionary.loadXml(xmlFile);
    }
}
